﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Channels;
using System.Threading;
using System.Net.Http;
using System.Threading.Tasks;

namespace Crawler
{
    static class SimpleCrawler
    {
        public class Content
        {
            public Uri url;
            public string htmlOrError;
        }
        static readonly HttpClient webClient = new()
        {
            Timeout = new(0, 0, 5)
        };
        static readonly Regex regex = new(@"(?:href|HREF)\s*=\s*[""']([^""']+\.(?:html|htm|aspx|jsp))[""']", RegexOptions.Compiled);

        public static async IAsyncEnumerable<string> Crawl(string startUrl, int fileAmount = 10)
        {
            yield return "开始爬行.... \r\n";
            yield return "爬行" + startUrl + "页面...\r\n";
            Uri entry = null;
            {
                string error = null;
                try
                {
                    entry = new(startUrl);
                }
                catch (UriFormatException e)
                {
                    error = e.Message;
                }
                if (error != null)
                {
                    yield return "爬行" + startUrl + "页面失败...\r\n" + error + "\r\n爬行结束\r\n";
                    yield break;
                }
            }
            if (fileAmount <= 0) fileAmount = 5;
            using CancellationTokenSource cancellSource = new();
            var token = cancellSource.Token;
            HashSet<Uri> urls = new();
            urls.Add(entry);
            int count = 1;
            BoundedChannelOptions options = new(9)
            {
                FullMode = BoundedChannelFullMode.Wait,
                SingleReader = true,
                SingleWriter = true
            };
            Channel<Uri> uri = Channel.CreateBounded<Uri>(options),
                mainUri = Channel.CreateBounded<Uri>(options);
            Channel<Content> content = Channel.CreateBounded<Content>(options),
                mainContent = Channel.CreateBounded<Content>(options);
            var uriReader = mainUri.Reader;
            var contentReader = mainContent.Reader;
            await uri.Writer.WriteAsync(entry);
            _ = DownLoad(uri, content, mainContent, token);
            _ = Parse(uri, content, mainUri, urls, token);

            bool uriReady, contentReady;
            do
            {
                uriReady = await uriReader.WaitToReadAsync(token);
                contentReady = await contentReader.WaitToReadAsync(token);
                if (uriReady)
                {
                    var baseUri = await uriReader.ReadAsync(token);
                    yield return "爬行" + baseUri.AbsoluteUri + "页面...\r\n";
                }
                if (contentReady)
                {
                    var info = await contentReader.ReadAsync(token);
                    var error = info.htmlOrError;
                    var baseUri = info.url;
                    if (error == null)
                    {
                        yield return "爬行" + baseUri.AbsoluteUri + "页面成功！\r\n";
                        if (++count >= fileAmount)
                        {
                            uri.Writer.TryComplete();
                            content.Writer.TryComplete();
                            cancellSource.Cancel();
                            break;
                        }
                    }
                    else
                    {
                        yield return "爬行" + baseUri?.AbsoluteUri + "页面失败...\r\n" + error + "\r\n";
                    }
                }
            } while (uriReady || contentReady);

            yield return "爬行结束\r\n";
        }

        public static async Task DownLoad(ChannelReader<Uri> uri, ChannelWriter<Content> html, ChannelWriter<Content> result, CancellationToken token)
        {
            while (!token.IsCancellationRequested && await uri.WaitToReadAsync(token))
            {
                Content content = new();
                try
                {
                    var url = await uri.ReadAsync(token);
                    content.url = url;
                    string htmlContent = await (await webClient.GetAsync(url, token)).Content.ReadAsStringAsync(token);
                    if (await html.WaitToWriteAsync(token)) await html.WriteAsync(new Content() { url = url, htmlOrError = htmlContent }, token);
                    string abs = url.AbsoluteUri;
                    string fileName = abs[(abs.LastIndexOf('/') + 1)..];
                    if (fileName.Equals(""))
                    {
                        fileName = "./index.html";
                    }
                    await File.WriteAllTextAsync(fileName, htmlContent, Encoding.UTF8, token);
                }
                catch (OperationCanceledException)
                {
                    break;
                }
                catch (Exception e)
                {
                    content.htmlOrError = e.Message;
                }
                if (await result.WaitToWriteAsync(token)) await result.WriteAsync(content, token);
            }
            result.Complete();
        }

        public static async Task Parse(ChannelWriter<Uri> uri, ChannelReader<Content> html, ChannelWriter<Uri> nextUri, HashSet<Uri> urls, CancellationToken token)
        {
            try
            {
                while (!token.IsCancellationRequested && await html.WaitToReadAsync(token))
                {
                    var content = await html.ReadAsync(token);
                    var matches = regex.Matches(content.htmlOrError);
                    var baseUri = content.url;
                    foreach (Match match in matches)
                    {
                        Uri strRef = new(baseUri, match.Groups[1].Value);
                        if (urls.Add(strRef))
                        {
                            if (await uri.WaitToWriteAsync(token)) await uri.WriteAsync(strRef, token);
                            if (await nextUri.WaitToWriteAsync(token)) await nextUri.WriteAsync(strRef, token);
                        }
                    }
                }
            }
            finally
            {
                nextUri.Complete();
            }
        }
    }
}
